{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "A=np.array([[11,12,13],[21,22,23],[31,32,33]])\n",
    "B=np.ones(shape=(3,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"add:0\", shape=(2,), dtype=int32)\n",
      "[3 6]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<bound method BaseSession.close of <tensorflow.python.client.session.Session object at 0x000001F5A9E4E4E0>>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a=tf.constant([1,2],name=\"a\")\n",
    "b=tf.constant([2,4],name=\"b\")\n",
    "result = a+b\n",
    "print(result)\n",
    "\n",
    "#上面只是定义了计算图，并没有运行计算图，所以不会输出运算结果\n",
    "\n",
    "sess=tf.Session()\n",
    "a=sess.run(result)\n",
    "print(a)\n",
    "sess.close\n",
    "\n",
    "#打开会话，运行计算图，关闭计算图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1  4  9 16]\n",
      "[ 1  4  9 16]\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    a=tf.constant([1,2,3,4])\n",
    "    b=tf.constant([1,2,3,4])\n",
    "    result=tf.multiply(a,b)\n",
    "    c=sess.run(result)\n",
    "    print(c)\n",
    "    \n",
    "print(c)\n",
    "\n",
    "# with 结束，计算会话自动关闭"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 3.95757794]]\n",
      "[[ 3.95757794]]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "#（2，3，1单元）3层前向神经网络（无激活函数）\n",
    "\n",
    "#生成服从标准差为1的正态分布的随机变量，作为初始化矩阵\n",
    "w1=tf.Variable(tf.random_normal([2,3],stddev=1,seed=1))\n",
    "w2=tf.Variable(tf.random_normal([3,1],stddev=1,seed=1))\n",
    "\n",
    "x=tf.constant([[0.7,0.9]])\n",
    "#矩阵乘法\n",
    "a=tf.matmul(x,w1)\n",
    "b=tf.matmul(a,w2)\n",
    "\n",
    "sess=tf.Session()\n",
    "#需要运行初始化赋值，前面只是定义，没运算\n",
    "sess.run(w1.initializer)\n",
    "sess.run(w2.initializer)\n",
    "y=sess.run(b)\n",
    "\n",
    "sess.close()\n",
    "print(y)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    w3=tf.Variable(tf.random_normal([2,3],stddev=1,seed=1))\n",
    "    w4=tf.Variable(tf.random_normal([3,1],stddev=1,seed=1))\n",
    "    x1=tf.constant([[0.7,0.9]])\n",
    "    a1=tf.matmul(x,w3)\n",
    "    b1=tf.matmul(a1,w4)\n",
    "    #可以直接嵌入初始化\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    print(sess.run(b1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 3.95757794]]\n",
      "[[ 3.95757794]\n",
      " [ 1.657197  ]\n",
      " [ 7.20209646]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<bound method BaseSession.close of <tensorflow.python.client.session.Session object at 0x000001F5A9F53F60>>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "w1=tf.Variable(tf.random_normal([2,3],stddev=1,seed=1))\n",
    "w2=tf.Variable(tf.random_normal([3,1],stddev=1,seed=1))\n",
    "\n",
    "#因为需要重复输入x，而每建一个x就会生成一个结点，计算图的效率会低。所以使用占位符\n",
    "x=tf.placeholder(tf.float32,shape=(1,2))\n",
    "x1=tf.placeholder(tf.float32,shape=(3,2))\n",
    "a=tf.matmul(x,w1)\n",
    "a1=tf.matmul(x1,w1)\n",
    "y=tf.matmul(a,w2)\n",
    "y1=tf.matmul(a1,w2)\n",
    "\n",
    "sess=tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "#运行y时将占位符填上，feed_dict为字典，变量名不可变\n",
    "y_hat=sess.run(y,feed_dict={x:[[0.7,0.9]]})\n",
    "y_hat1=sess.run(y1,feed_dict={x1:[[0.7,0.9],[0.2,0.5],[1,2]]})# batch = 3\n",
    "print(y_hat)\n",
    "print(y_hat1)\n",
    "sess.close\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.81131822  1.48459876  0.06532937]\n",
      " [-2.4427042   0.0992484   0.59122431]]\n",
      "[[-0.81131822]\n",
      " [ 1.48459876]\n",
      " [ 0.06532937]]\n",
      "在迭代 0 次后，训练损失为 0.047106\n",
      "在迭代 1000 次后，训练损失为 0.0114981\n",
      "在迭代 2000 次后，训练损失为 0.00481489\n",
      "在迭代 3000 次后，训练损失为 0.00285046\n",
      "在迭代 4000 次后，训练损失为 0.00207789\n",
      "在迭代 5000 次后，训练损失为 0.00158399\n",
      "在迭代 6000 次后，训练损失为 0.0011365\n",
      "在迭代 7000 次后，训练损失为 0.000963961\n",
      "在迭代 8000 次后，训练损失为 0.000769849\n",
      "在迭代 9000 次后，训练损失为 0.000558014\n",
      "在迭代 10000 次后，训练损失为 0.000354686\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from numpy.random import RandomState\n",
    "\n",
    "batch_size=10\n",
    "w1=tf.Variable(tf.random_normal([2,3],stddev=1,seed=1))\n",
    "w2=tf.Variable(tf.random_normal([3,1],stddev=1,seed=1))\n",
    "\n",
    "# None 可以根据batch 大小确定维度，在shape的一个维度上使用None，方便不大的batch\n",
    "x=tf.placeholder(tf.float32,shape=(None,2))\n",
    "y=tf.placeholder(tf.float32,shape=(None,1))\n",
    "\n",
    "a=tf.matmul(x,w1)\n",
    "yhat=tf.matmul(a,w2)\n",
    "\n",
    "#定义交叉熵为损失函数，训练过程使用Adam算法最小化交叉熵\n",
    "cross_entropy=-tf.reduce_mean(y*tf.log(tf.clip_by_value(yhat,1e-10,1.0)))\n",
    "train_step=tf.train.AdamOptimizer(0.001).minimize(cross_entropy)\n",
    "\n",
    "rdm=RandomState(1)\n",
    "data_size=516\n",
    "\n",
    "#生成两个特征，共data_size个样本\n",
    "X=rdm.rand(data_size,2)\n",
    "#定义规则给出样本标签，所有x1+x2<1的样本认为是正样本，其他为负样本。Y，1为正样本\n",
    "Y = [[int(x1+x2 < 1)] for (x1, x2) in X]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    print(sess.run(w1))\n",
    "    print(sess.run(w2))\n",
    "    steps=11000\n",
    "    for i in range(steps):\n",
    "        \n",
    "        #选定每一个批量读取的首尾位置，确保在1个epoch内采样训练\n",
    "        start = i * batch_size % data_size\n",
    "        end = min(start + batch_size,data_size)\n",
    "        sess.run(train_step,feed_dict={x:X[start:end],y:Y[start:end]})\n",
    "        if i % 1000 == 0:\n",
    "            training_loss= sess.run(cross_entropy,feed_dict={x:X,y:Y})\n",
    "            print(\"在迭代 %d 次后，训练损失为 %g\"%(i,training_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### 激活函数和偏置项：\n",
    "a=tf.nn.relu(tf.matmul(x,w1)+biases1)\n",
    "\n",
    "yhat=tf.nn.relu(tf.matmul(a,w2)+biases2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### 交叉熵函数\n",
    "cross_entropy=-tf.reduce_mean(y*tf.log(tf.clip_by_value(yhat,1e-10,1.0)))\n",
    "\n",
    "tf.reduce_mean(x)表示计算全局平均值。tf.clip_by_value()函数可以将张量中的数值限制在一个范围内。tf.log()对张量内的所有元素依次求对数。交叉熵函数一般会与softmax回归一起使用，TensorFlow将它们进行了统一封装：cross_entropy=tf.nn.softmax_cross_entropy_with_logits(yhat,y)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 均方误差损失函数\n",
    "mse=tf.reduce_mean(tf.square(y-yhat))\n",
    "\n",
    "tf.select(tf.greater(y,yhat),y-yhat,yhat-y)\n",
    "\n",
    "tf.greater()的输入是两个张量，比较两个张量中的每一个元素，并返回比较结果（true或false的向量）。tf.select()有三个参数，第一个参数条件为真时选择第二个参数中的值，否则选择第三个参数的值。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#decayed_learning_rate=learning_rate*decay_rate^(global_steps/decay_steps)，指数衰减函数的定义\n",
    "\n",
    "global_step=tf.Variable(0)\n",
    "#使用exponential_decay生成学习速率，因为staircase=tire，每100次迭代，学习率×0.96\n",
    "learning_rate=tf.train.exponential_decay(0.1,global_step,100,0.96,staircase=True)\n",
    "#在minimize中导入global_step将自动更新\n",
    "#learning_step=tf.train.GtadientDescentOptimizer(learning_rate).minimize(loss_function,global_step=global_step)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### 带L2正则化的损失函数\n",
    "w=tf.Variable(tf.random_normal([2,1],stddev=1,seed=1))\n",
    "\n",
    "yhat=tf.matmul(x,w)\n",
    "\n",
    "loss=tf.reduce_mean(tf.square(y-yhat))+tf.contrib.layers.l2_regularizer(lambda)(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.5\n",
      "12.75\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "w=tf.constant([[1.0,-5.0],[-3.0,4.0]])\n",
    "with tf.Session() as sess:\n",
    "    #L1正则化：(1+5+3+4)×0.5\n",
    "    print(sess.run(tf.contrib.layers.l1_regularizer(0.5)(w)))\n",
    "    #L2正则化：（1+25+9+16）/2×0.5，L2正则化会处以2，无偏估计？\n",
    "    print(sess.run(tf.contrib.layers.l2_regularizer(0.5)(w)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "神经网络结构复杂后，定义网络结构的部分和计算损失函数的部分可能不在同一个函数中。所以采用collection在一个计算图中保留一组实体（如张量）。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "#通过集合（collection）计算一个5层神经网络带L2正则化的损失函数\n",
    "\n",
    "#随机正态初始化一层神经网络的权重，并将权重的L2正则化损失加入名为losses的集合中，返回初始化的权重\n",
    "def get_weight(shape,lambd):\n",
    "    var=tf.Variable(tf.random_normal(shape),dtype=tf.float32)\n",
    "    \n",
    "    #tf.add_to_collection函数将新生成变量的L2正则化损失项加入集合，第一个参数为集合名，第二个参数为加入集合的内容。\n",
    "    tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(lambd)(var))\n",
    "    return var\n",
    "\n",
    "x=tf.placeholder(tf.float32,shape=(None,2))\n",
    "y=tf.placeholder(tf.float32,shape=(None,1))\n",
    "batch_size=8\n",
    "\n",
    "#定义每一层中结点个数和层数\n",
    "layer_dimension=[2,10,10,10,1]\n",
    "n_layers=len(layer_dimension)\n",
    "\n",
    "#该变量维护前向传播时最深层的结点，最开始为输入层\n",
    "cur_layer=x\n",
    "\n",
    "#输入层结点个数\n",
    "in_dimension=layer_dimension[0]\n",
    "\n",
    "#通过循环生成5层全连接神经网络\n",
    "for i in range(1,n_layers):\n",
    "    \n",
    "    #下一层节点数\n",
    "    out_dimension=layer_dimension[i]\n",
    "    \n",
    "    #生成当前层中权重的变量，并将这个变量的L2正则化损失加入计算图上的集合\n",
    "    #[in_dimension,out_dimension],例第一层到第二层之间的权重维度为2×10\n",
    "    weight=get_weight([in_dimension,out_dimension],0.001)\n",
    "    \n",
    "    #偏置项和后一层维度相等，为什么是wx+0.1，而不是wx+b？？\n",
    "    bias=tf.Variable(tf.constant(0.1,shape=[out_dimension]))\n",
    "    \n",
    "    #使用ReLU激活函数，cur_layer储存传播一层后的激活情况，后一层激活函数的输出\n",
    "    cur_layer=tf.nn.relu(tf.matmul(cur_layer,weight)+bias)\n",
    "    in_dimension=layer_dimension[i]\n",
    "    \n",
    "mse_loss=tf.reduce_mean(tf.square(y-cur_layer))\n",
    "\n",
    "#将均方误差函数加入损失集合\n",
    "tf.add_to_collection('losses',mse_loss)\n",
    "\n",
    "#get_collection返回一个列表，这个列表是集合中的所有元素，这些元素就是组成损失函数的误差和正则项，相加得最终损失函数\n",
    "loss=tf.add_n(tf.get_collection('losses'))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n",
      "10.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "#tf.assign(A, new_number),这个函数的功能主要是把A的值变为new_number\n",
    "A=tf.Variable(tf.constant(0.0),dtype=tf.float32)\n",
    "with tf.Session() as sess:  \n",
    "    sess.run(tf.global_variables_initializer())  \n",
    "    print (sess.run(A))  \n",
    "    sess.run(tf.assign(A, 10))  \n",
    "    print (sess.run(A))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.0, 0.0]\n",
      "[5.0, 4.5]\n",
      "[10.0, 4.5549998]\n",
      "[10.0, 4.6094499]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "#滑动平均模型\n",
    "\n",
    "#定义一个变量计算滑动平均，初始值为0，所有需要计算滑动平均的变量必须是实数型\n",
    "v1=tf.Variable(0,dtype=tf.float32)\n",
    "\n",
    "#step变量模拟神经网络中的迭代次数，用于动态控制衰减率\n",
    "step=tf.Variable(0,trainable=False)\n",
    "\n",
    "#定义一个滑动平均的类，初始化时给定了衰减率和控制衰减率的变量step\n",
    "ema=tf.train.ExponentialMovingAverage(0.99,step)\n",
    "\n",
    "#定义一个更新变量的滑动平均操作，给定一个列表，每次执行操作时更新列表所有变量\n",
    "maintain_averages_op=ema.apply([v1])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    \n",
    "    #初始化所有变量\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    #通过ema.average(v1)获取滑动平均之后变量的取值。初始化后，v1的值和滑动平均都为0\n",
    "    print(sess.run([v1,ema.average(v1)]))\n",
    "    \n",
    "    #更新变量v1的值为5,tf.assign将数值分配给变量\n",
    "    sess.run(tf.assign(v1,5))\n",
    "    \n",
    "    #更新v1的滑动平均值。衰减率为min{0.99,(1+step)/(10+step)≈0.1}=0.1，所以v1的滑动平均值会更新为 0.1×0+0.9×5=4.5\n",
    "    sess.run(maintain_averages_op)\n",
    "    print(sess.run([v1,ema.average(v1)]))\n",
    "    \n",
    "    #将迭代设置为10000步\n",
    "    sess.run(tf.assign(step,10000))\n",
    "    \n",
    "    #更新v1的值为10\n",
    "    sess.run(tf.assign(v1,10))\n",
    "    \n",
    "    #更新v1的滑动平均值。衰减率为min{0.99,(1+step)/(10+step)≈0.999}=0.99，所以v1的滑动平均会被更新为0.99×4.5+0.01×10=4.555\n",
    "    sess.run(maintain_averages_op)\n",
    "    print(sess.run([v1,ema.average(v1)]))\n",
    "    \n",
    "    #再次更新滑动平均值，得到新的滑动平均值为0.99×4.555+0.01×10=4.60945\n",
    "    sess.run(maintain_averages_op)\n",
    "    print(sess.run([v1,ema.average(v1)]))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### MNIST 手写字体识别"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./data/MNIST/train-images-idx3-ubyte.gz\n",
      "Extracting ./data/MNIST/train-labels-idx1-ubyte.gz\n",
      "Extracting ./data/MNIST/t10k-images-idx3-ubyte.gz\n",
      "Extracting ./data/MNIST/t10k-labels-idx1-ubyte.gz\n",
      "Training data size:  55000\n",
      "Validating data size:  5000\n",
      "Testing data size:  10000\n"
     ]
    }
   ],
   "source": [
    "#导入数据\n",
    "import tensorflow as tf\n",
    "\n",
    "#原网站提供了6W张训练图片和1W张测试图片，导入的该工具会从训练图片分出5000张作为验证集\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "#读取路径为当前路径下的data文件夹下的MNIST文件夹内，如果该文件夹没有，则自动下载数据至该文件夹\n",
    "mnist = input_data.read_data_sets(\"./data/MNIST/\", one_hot=True)\n",
    "\n",
    "print(\"Training data size: \", mnist.train.num_examples) \n",
    "print (\"Validating data size: \", mnist.validation.num_examples) \n",
    "print (\"Testing data size: \", mnist.test.num_examples) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X shapr: (100, 784)\n",
      "Y shape: (100, 10)\n"
     ]
    }
   ],
   "source": [
    "#为了方便使用SGD，mnist.train.next_batch函数可以从所有训练数据中取一个小批量投入训练\n",
    "\n",
    "batch_size=100\n",
    "\n",
    "#从训练集选取batch_size个训练数据\n",
    "xs,ys=mnist.train.next_batch(batch_size)\n",
    "\n",
    "#将图片展开成一个长度为28×28=784的一维数组，一张图片可作为一个特征向量。所以batch为100的矩阵维度为100×784\n",
    "print('X shapr:',xs.shape)\n",
    "print('Y shape:',ys.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./data/MNIST/train-images-idx3-ubyte.gz\n",
      "Extracting ./data/MNIST/train-labels-idx1-ubyte.gz\n",
      "Extracting ./data/MNIST/t10k-images-idx3-ubyte.gz\n",
      "Extracting ./data/MNIST/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"./data/MNIST/\", one_hot=True)\n",
    "\n",
    "\n",
    "#输入结点数为像素点数，输出结点数为类别数\n",
    "INPUT_NODE=784\n",
    "OUTPUT_NODE=10\n",
    "\n",
    "#一个隐藏层\n",
    "LAYER1_NODE=500\n",
    "\n",
    "#一个批量中的样本量，数据量越小训练过程越接近SGD，数据量越大训练过程越接近梯度下降\n",
    "BATCH_SIZE=100\n",
    "\n",
    "#学习率和学习衰减率\n",
    "LEARNING_RATE_BASE=0.8\n",
    "LEARNING_RATE_DECAY=0.99\n",
    "\n",
    "#正则化系数、迭代次数和滑动平均衰减率\n",
    "REGULARIZATION_RATE=0.0001\n",
    "TRAINING_STEPS=3000\n",
    "MOVING_AVERAGE_DECAY=0.99\n",
    "\n",
    "#定义推断函数，给定所有参数下计算神经网络的前向传播结果。参数avg_class可确定推断中使不使用滑动平均模型\n",
    "def inference(input_tensor,avg_class,weights1,biases1,weights2,biases2):\n",
    "    \n",
    "    #没有提供滑动平均类时，直接使用参数当前的取值\n",
    "    if avg_class == None:\n",
    "        \n",
    "        #计算隐藏层前向传播结果，使用ReLU激活函数\n",
    "        layer1=tf.nn.relu(tf.matmul(input_tensor,weights1)+biases1)\n",
    "        \n",
    "        #计算输出层的前向传播结果\n",
    "        return tf.matmul(layer1,weights2)+biases2\n",
    "    else:\n",
    "        \n",
    "        #首先使用avg_class.averaage函数计算变量的滑动均值，然后计算相应的前向传播结果\n",
    "        layer1=tf.nn.relu(tf.matmul(input_tensor,avg_class.average(weights1))+avg_class.average(biases1))\n",
    "        return tf.matmul(layer1,avg_class.average(weights2))+avg_class.average(biases2)\n",
    "    \n",
    "#模型训练函数\n",
    "\n",
    "def train(mnist):\n",
    "    x=tf.placeholder(tf.float32,[None,INPUT_NODE],name='x-input')\n",
    "    y=tf.placeholder(tf.float32,[None,OUTPUT_NODE],name='y-input')\n",
    "    \n",
    "    #生成隐藏层参数\n",
    "    weights1=tf.Variable(tf.truncated_normal([INPUT_NODE,LAYER1_NODE],stddev=0.1))\n",
    "    biases1=tf.Variable(tf.constant(0.1,shape=[LAYER1_NODE]))\n",
    "    \n",
    "    #生成输出层参数\n",
    "    weights2=tf.Variable(tf.truncated_normal([LAYER1_NODE,OUTPUT_NODE],stddev=0.1))\n",
    "    biases2=tf.Variable(tf.constant(0.1,shape=[OUTPUT_NODE]))\n",
    "    \n",
    "    #计算当前参数下前向传播的结果，这里设为‘None’不会计算滑动平均值\n",
    "    y_hat=inference(x,None,weights1,biases1,weights2,biases2)\n",
    "\n",
    "    #定义储存迭代数的变量，这个变量不需要计算滑动平均值，所以这里指定的这个变量为不饿训练变量（trainable=False）\n",
    "    global_step=tf.Variable(0,trainable=False)\n",
    "    \n",
    "    #给定滑动平均衰减率和迭代数，初始化滑动平均类。\n",
    "    variable_averages=tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY,global_step)\n",
    "    \n",
    "    #在所有代表神经网络参数的变量上使用滑动平均，其他超参数不需要。tf.trainable_variables返回的就是图上的集合GraphKeys.TRAINABLE_VARIABLES中的元素。\n",
    "    variables_averages_op=variable_averages.apply(tf.trainable_variables())\n",
    "    \n",
    "    #计算使用滑动平均后的前向传播结果，滑动平均不会改变变量本身，而是使用影子变量记录滑动平均值，需要使用滑动平均再明确调用average函数\n",
    "    average_y_hat=inference(x,variable_averages,weights1,biases1,weights2,biases2)\n",
    "    \n",
    "    #~使用tf.argmax函数得到正确答案对应的类别编号\n",
    "    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y, 1))\n",
    "    \n",
    "    #计算当前批量中所有样本的交叉熵均值\n",
    "    cross_entropy_mean=tf.reduce_mean(cross_entropy)\n",
    "    \n",
    "    #计算L2正则化损失函数\n",
    "    regularizer=tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)\n",
    "    \n",
    "    #计算模型的正则化损失，只计算神经网络权重的正则化损失，不使用偏置项\n",
    "    regularization=regularizer(weights1)+regularizer(weights2)\n",
    "    \n",
    "    #总损失函数\n",
    "    loss=cross_entropy_mean+regularization\n",
    "    \n",
    "    #设置指数衰减学习率.基础学习率、当前迭代次数、一个epoch所需要的迭代次数、学习衰减率\n",
    "    learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,global_step,mnist.train.num_examples/BATCH_SIZE,LEARNING_RATE_DECAY)\n",
    "    \n",
    "    #使用梯度下降优化算法优化损失函数，损失函数包括交叉熵损失和L2正则化损失\n",
    "    train_step=tf.train.GradientDescentOptimizer(learning_rate).minimize(loss,global_step=global_step)\n",
    "    \n",
    "    #在训练神经网络模型时，每过一遍数据既需要通过反向传播来更新参数，也要更新每个参数的滑动平均值。为了一次完成多个操作\n",
    "    #train_op=tf.group(train_step,variables_averages_op)\n",
    "    with tf.control_dependencies([train_step,variables_averages_op]):\n",
    "        train_op=tf.no_op(name='train')\n",
    "        \n",
    "    correct_prediction=tf.equal(tf.argmax(average_y_hat,1),tf.argmax(y,1))\n",
    "    accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "    #初始化会话并开始训练过程\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        validate_feed={x:mnist.validation.images,y:mnist.validation.labels}\n",
    "    \n",
    "        test_feed={x:mnist.test.images,y:mnist.test.labels}\n",
    "    \n",
    "        # 循环的训练神经网络。\n",
    "        for i in range(TRAINING_STEPS):\n",
    "            if i % 1000 == 0:\n",
    "                validate_acc = sess.run(accuracy, feed_dict=validate_feed)\n",
    "                print(\"After %d training step(s), validation accuracy using average model is %g \" % (i, validate_acc))\n",
    "            \n",
    "            xs,ys=mnist.train.next_batch(BATCH_SIZE)\n",
    "            sess.run(train_op,feed_dict={x:xs,y:ys})\n",
    "\n",
    "        test_acc=sess.run(accuracy,feed_dict=test_feed)\n",
    "        print((\"After %d training step(s), test accuracy using average model is %g\" %(TRAINING_STEPS, test_acc)))\n",
    "\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "avg_class = None\n",
    "train(mnist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
